# -*- coding: utf-8 -*-
import scrapy

from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import HtmlXPathSelector

from stack.items import StackItem
from scrapy.conf import settings


class StackSpider(CrawlSpider):
    name = "stacktags1"
    allowed_domains = ["stackoverflow.com"]
    start_urls = [
        "http://stackoverflow.com/questions/tagged/java?sort=votes&pagesize=50&page=1"
    ]
    rules = (
        Rule(
            SgmlLinkExtractor(allow=('&page=\d')),
            callback='parse',
            follow=True
        ),
    )
    g_counter = 0

    def start_requests(self):
        urls = []
        for page_index in range(13956, 27913):
            url = 'http://stackoverflow.com/questions/tagged/java?sort=votes&pagesize=50&page=' + str(page_index)
            urls.append(url)

        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        hxs = HtmlXPathSelector(response)
        questions = hxs.xpath('.//div[@class="summary"]')
        counter = 0
        for question in questions:
            item = StackItem()
            item['title'] = question.xpath(
                '//h3//a[@class="question-hyperlink"]/text()').extract()[counter]
            item['url'] = question.xpath(
                '//h3//a[@class="question-hyperlink"]/@href').extract()[counter]
            item['tags'] = ";".join(question.xpath('div/a[@class="post-tag"]/text()').extract())
            
            counter = counter + 1
            print 'No.' + str(counter) + 'counter.' + item['tags'] + item['title'] + item['url'] 
            yield item